### this must be run with web2py.py -S vis -M -R thisscript.py
### settigns are defined in vis/models/0.py

import time
import os
import sys
import subprocess
import logging

# job status QUEUED > PROCESSING > COMPLETED

logging.basicConfig(filename=settings.pbs_log,level=settings.debuglevel)

serialize=False # for debugging
lock=False

while(True):
    jobs = db(db.job.status != 'completed').select()
    for job in jobs:
        logging.info('processing job #%i' % job.id)
        script_file = os.path.join(request.folder,'uploads',job.script)
        stdout_file = script_file + '.out'
        stderr_file = script_file + '.err'
        if job.status == QUEUED and not (lock and serialize):
            lock=True
            print 'submitting id %d: %s %s %s' % \
                (job.id, script_file, stdout_file, stderr_file)
            logging.info('submitting id %d: %s %s %s' % \
                             (job.id, script_file, stdout_file, stderr_file))
            p = subprocess.Popen(args=('qsub',
                                       '-e', stderr_file,
                                       '-o', stdout_file,
                                       script_file),
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            p.wait()
            output = p.stdout.readlines()
            job_id = output[0].strip()
            logging.info('queued with job_id=%s' % job_id)
            job.update_record(status=PROCESSING,job_id=job_id)
            db.commit()
        elif job.status == PROCESSING:
            p = subprocess.Popen(args=['qstat',job.job_id],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            p.wait()
            output = p.stdout.readlines()[0].strip()
            if output == 'qstat: Unknown Job Id %s' % job.job_id:
                logging.info('job id %s is no longer running' % job.job_id)
                job.update_record(status=COMPLETED,
                                  stdout=stdout_file,
                                  stderr=stderr_file)
                db.commit()
                lock=False
            else:
                logging.info('job id %s is still running' % job.job_id)
    time.sleep(5)
